{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fold 1, train loss 0.002749, valid loss 0.000194\n",
      "fold 2, train loss 0.003578, valid loss 0.000232\n",
      "fold 3, train loss 0.003604, valid loss 0.000236\n",
      "fold 4, train loss 0.002041, valid loss 0.000147\n",
      "fold 5, train loss 0.002485, valid loss 0.000180\n",
      "fold 6, train loss 0.002277, valid loss 0.000169\n",
      "fold 7, train loss 0.004013, valid loss 0.000252\n",
      "fold 8, train loss 0.002703, valid loss 0.000194\n",
      "fold 9, train loss 0.002221, valid loss 0.000156\n",
      "fold 10, train loss 0.002759, valid loss 0.000204\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.utils.data\n",
    "import torch.nn.init\n",
    "import numpy as np\n",
    "\n",
    "n_train = 7000\n",
    "n_test = 3000\n",
    "num_inputs = 500\n",
    "true_w = torch.ones(num_inputs, 1) * 0.0056\n",
    "true_b = 0.028\n",
    "\n",
    "features = torch.randn(n_train + n_test, num_inputs)\n",
    "labels = torch.matmul(features, true_w) + true_b\n",
    "labels += torch.tensor(np.random.normal(0, 0.01, size=labels.size()), dtype= torch.float)\n",
    "\n",
    "batch_size = 200\n",
    "num_epochs = 20\n",
    "lr = 0.01\n",
    "\n",
    "\n",
    "def squared_loss(y_hat, y):\n",
    "    return (y_hat - y.view(y_hat.size())) ** 2 / 2\n",
    "loss = squared_loss\n",
    "\n",
    "# 训练模型\n",
    "def train(x_train, y_train, x_valid, y_valid):\n",
    "    net = nn.Linear(num_inputs, 1)\n",
    "    nn.init.normal_(net.weight, mean=0, std=0.1)\n",
    "    nn.init.normal_(net.bias, mean=0, std=1)\n",
    "    optimizer_w = torch.optim.SGD(params=[net.weight], lr=lr)\n",
    "    optimizer_b = torch.optim.SGD(params=[net.bias], lr=lr)\n",
    "\n",
    "    train_dataset = torch.utils.data.TensorDataset(x_train,y_train)\n",
    "    train_dataiter = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True, num_workers=0)\n",
    "\n",
    "    valid_dataset = torch.utils.data.TensorDataset(x_valid,y_valid)\n",
    "    valid_dataiter = torch.utils.data.DataLoader(dataset=valid_dataset, batch_size=batch_size, shuffle=False, num_workers=0)\n",
    "\n",
    "    train_ls = []\n",
    "    test_ls = []\n",
    "    n, train_loss_sum = 0, 0\n",
    "    n_test, test_loss_sum = 0, 0\n",
    "\n",
    "    for epoch in range(num_epochs):\n",
    "        for X, y in train_dataiter:\n",
    "            l = loss(net(X), y).mean()\n",
    "            optimizer_b.zero_grad()\n",
    "            optimizer_w.zero_grad()\n",
    "            l.backward()\n",
    "            optimizer_w.step()\n",
    "            optimizer_b.step()\n",
    "            n += y.shape[0]\n",
    "            train_loss_sum +=l.item()\n",
    "        train_ls.append(train_loss_sum / n)\n",
    "\n",
    "        for X, y in valid_dataiter:\n",
    "            l = loss(net(X), y).mean()\n",
    "            n_test += y.shape[0]\n",
    "            test_loss_sum += l.item()\n",
    "        test_ls.append(test_loss_sum / n)\n",
    "\n",
    "        # print('epoch %d, w= %.4f, b= %.3f' % (epoch + 1, net.weight.mean().item(), net.bias.mean().item()))\n",
    "    return train_ls, test_ls\n",
    "\n",
    "\n",
    "def get_k_fold_data(k, i, X, y):\n",
    "    # 返回第i折交叉验证时所需要的训练和验证数据，分开放，X_train为训练数据，X_valid为验证数据\n",
    "    assert k > 1\n",
    "    fold_size = X.shape[0] // k  \n",
    "    X_train, y_train = None, None\n",
    "    for j in range(k):\n",
    "        idx = slice(j * fold_size, (j + 1) * fold_size)  #slice(start,end,step)切片函数\n",
    "        X_part, y_part = X[idx, :], y[idx]\n",
    "        if j == i:\n",
    "            X_valid, y_valid = X_part, y_part\n",
    "        elif X_train is None:\n",
    "            X_train, y_train = X_part, y_part\n",
    "        else:\n",
    "            X_train = torch.cat((X_train, X_part), dim=0)\n",
    "            y_train = torch.cat((y_train, y_part), dim=0)\n",
    "    return X_train, y_train, X_valid, y_valid\n",
    "\n",
    "\n",
    "def k_fold(k, X_train, y_train, num_epochs, batch_size):\n",
    "    for i in range(k):\n",
    "        data = get_k_fold_data(k, i, X_train, y_train) # 获取k折交叉验证的训练和验证数据\n",
    "        train_ls, valid_ls = train(*data)\n",
    "        train_l_sum, valid_l_sum, train_acc_sum, valid_acc_sum = 0.0, 0.0, 0.0, 0.0\n",
    "        for ii in train_ls:\n",
    "            train_l_sum += ii\n",
    "        for jj in valid_ls:\n",
    "            valid_l_sum += jj\n",
    "\n",
    "        print('fold %d, train loss %.6f, valid loss %.6f' % (i + 1, train_l_sum / num_epochs, valid_l_sum / num_epochs))\n",
    "\n",
    "\n",
    "k_fold(10, features, labels, num_epochs, batch_size)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fold 1, train loss 0.0106, valid loss 0.0010, train acc 0.9993, test acc 1.0000\n",
      "fold 2, train loss 0.0106, valid loss 0.0010, train acc 0.9992, test acc 1.0000\n",
      "fold 3, train loss 0.0106, valid loss 0.0010, train acc 0.9996, test acc 1.0000\n",
      "fold 4, train loss 0.0106, valid loss 0.0010, train acc 0.9994, test acc 1.0000\n",
      "fold 5, train loss 0.0109, valid loss 0.0010, train acc 0.9990, test acc 1.0000\n",
      "fold 6, train loss 0.0107, valid loss 0.0010, train acc 0.9990, test acc 1.0000\n",
      "fold 7, train loss 0.0107, valid loss 0.0010, train acc 0.9983, test acc 1.0000\n",
      "fold 8, train loss 0.0112, valid loss 0.0010, train acc 0.9981, test acc 1.0000\n",
      "fold 9, train loss 0.0105, valid loss 0.0010, train acc 0.9997, test acc 1.0000\n",
      "fold 10, train loss 0.0107, valid loss 0.0010, train acc 0.9989, test acc 1.0000\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import matplotlib.pyplot as plt\n",
    "import random\n",
    "\n",
    "#数据集\n",
    "train1 = torch.normal(-1, 1, size=(7000, 200))\n",
    "test1 = torch.normal(-1, 1, size=(3000, 200))\n",
    "train2 = torch.normal(1, 1, size=(7000, 200))\n",
    "test2 = torch.normal(1, 1, size=(3000, 200))\n",
    "label_train1 = torch.zeros(7000)\n",
    "label_train2 = torch.ones(7000)\n",
    "label_test1 = torch.zeros(3000)\n",
    "label_test2 = torch.ones(3000)\n",
    "\n",
    "train_features = torch.cat((train1, train2), 0).type(torch.FloatTensor)\n",
    "train_labels = torch.cat((label_train1, label_train2), 0)\n",
    "test_features = torch.cat((test1, test2), 0).type(torch.FloatTensor)\n",
    "test_labels = torch.cat((label_test1, label_test2), 0)\n",
    "\n",
    "\n",
    "def data_iter(batch_size,features,labels,shuffle):\n",
    "    num_examples = len(features)\n",
    "    indices = list(range(num_examples))\n",
    "    if shuffle == True:\n",
    "        random.shuffle(indices)\n",
    "    for i in range(0, num_examples, batch_size):\n",
    "        j = torch.LongTensor(indices[i:min(i+batch_size,num_examples)])\n",
    "        yield features.index_select(0,j),labels.index_select(0,j)\n",
    "\n",
    "def relu(x):\n",
    "    return torch.max(input = x, other=torch.tensor(0.0))\n",
    "\n",
    "def sigmoid(x):\n",
    "    return 1.0/(1+torch.exp(-x))\n",
    "\n",
    "def crossloss(output, label):\n",
    "    output=output.squeeze(-1)\n",
    "    return -(label*torch.log(output)+(1-label)*torch.log(1-output))\n",
    "\n",
    "def sgd(params,lr,batch_size):\n",
    "    for param in params:\n",
    "        param.data-=lr*param.grad/batch_size\n",
    "        param.grad.zero_()\n",
    "\n",
    "num_inputs = 200\n",
    "num_hidden = 100\n",
    "num_outputs = 1\n",
    "def train(batch_size, num_epochs, lr, train_features, train_labels, test_features, test_labels):\n",
    "    train_ls, test_ls, x_epoch, list_train_acc, list_test_acc=[], [], [], [], []\n",
    "    w1 = torch.normal(0, 0.01, size=(num_inputs, num_hidden), requires_grad=True)\n",
    "    b1 = torch.zeros(num_hidden, requires_grad=True)\n",
    "    w2 = torch.normal(0, 0.01, size=(num_hidden, num_outputs), requires_grad=True)\n",
    "    b2 = torch.zeros(num_outputs, requires_grad=True)\n",
    "\n",
    "    for epoch in range(num_epochs):\n",
    "        train_loss_sum, train_num, train_acc = 0, 0, 0\n",
    "        test_loss_sum,test_num,test_acc=0, 0, 0\n",
    "\n",
    "        for x,y in data_iter(batch_size, train_features, train_labels, True):\n",
    "            hidden = relu(torch.mm(x, w1) + b1)\n",
    "            output = sigmoid(torch.mm(hidden, w2) + b2)\n",
    "            loss = crossloss(output, y).sum()\n",
    "            train_loss_sum += loss.item()\n",
    "            output_temp = output.squeeze(-1)     \n",
    "            train_acc += (output_temp.gt(0.5) == y).sum().item()\n",
    "            train_num += x.shape[0]\n",
    "            loss.backward()\n",
    "            sgd([w1, b1, w2, b2], lr, batch_size)\n",
    "        train_ls.append(train_loss_sum/train_num)\n",
    "        x_epoch.append(epoch + 1)\n",
    "\n",
    "        for x,y in data_iter(batch_size, test_features, test_labels, False):\n",
    "            hidden = relu(torch.mm(x, w1) + b1)\n",
    "            output = sigmoid(torch.mm(hidden, w2) + b2)\n",
    "            loss = crossloss(output, y).sum()\n",
    "            test_loss_sum += loss.item()\n",
    "            output_temp = output.squeeze(-1)\n",
    "            test_acc += (output_temp.gt(0.5) == y).sum().item()\n",
    "            test_num += x.shape[0]\n",
    "        test_ls.append(test_loss_sum/test_num)\n",
    "        list_train_acc.append(train_acc/train_num)\n",
    "        list_test_acc.append(test_acc/test_num)\n",
    "        # print(\"epoch %d,train loss %f,train acc %f,test loss %f,test acc %f\"%(epoch + 1, train_loss_sum/train_num, train_acc/train_num, test_loss_sum/test_num, test_acc/test_num))\n",
    "\n",
    "    return train_ls, test_ls, list_train_acc, list_test_acc\n",
    "# train(batch_size, num_epochs, lr, train_features, train_labels, test_features, test_labels)\n",
    "\n",
    "def get_k_fold_data(k, i, X, y):\n",
    "    # 返回第i折交叉验证时所需要的训练和验证数据，分开放，X_train为训练数据，X_valid为验证数据\n",
    "    assert k > 1\n",
    "    fold_size = X.shape[0] // k  # 双斜杠表示除完后再向下取整\n",
    "    X_train, y_train = None, None\n",
    "    for j in range(k):\n",
    "        idx = slice(j * fold_size, (j + 1) * fold_size)  #slice(start,end,step)切片函数\n",
    "        X_part, y_part = X[idx, :], y[idx]\n",
    "        if j == i:\n",
    "            X_valid, y_valid = X_part, y_part\n",
    "        elif X_train is None:\n",
    "            X_train, y_train = X_part, y_part\n",
    "        else:\n",
    "            X_train = torch.cat((X_train, X_part), dim=0)\n",
    "            y_train = torch.cat((y_train, y_part), dim=0)\n",
    "    return X_train, y_train, X_valid, y_valid\n",
    "\n",
    "def k_fold(k, X_train, y_train, num_epochs, batch_size):\n",
    "    for i in range(k):\n",
    "        data = get_k_fold_data(k, i, X_train, y_train) # 获取k折交叉验证的训练和验证数据\n",
    "        train_ls, valid_ls, train_acc, valid_acc = train(batch_size, num_epochs, lr, *data)\n",
    "        train_l_sum, valid_l_sum, train_acc_sum, valid_acc_sum = 0.0, 0.0, 0.0, 0.0\n",
    "        for ii in train_ls:\n",
    "            train_l_sum += ii\n",
    "        for jj in valid_ls:\n",
    "            valid_l_sum += jj\n",
    "        for ii in train_acc:\n",
    "            train_acc_sum +=ii\n",
    "        for jj in valid_acc:\n",
    "            valid_acc_sum +=jj\n",
    "        print('fold %d, train loss %.4f, valid loss %.4f, train acc %.4f, test acc %.4f' % (i + 1, train_l_sum / num_epochs, valid_l_sum / num_epochs, train_acc_sum / num_epochs, valid_acc_sum / num_epochs))\n",
    "\n",
    "batch_size = 256\n",
    "num_epochs = 10\n",
    "lr = 0.1\n",
    "fold = 10\n",
    "k_fold(fold, train_features, train_labels, num_epochs, batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fold 1, train loss 0.0007, valid loss 0.0007, train acc 0.9598, test acc 0.9612\n",
      "fold 2, train loss 0.0007, valid loss 0.0007, train acc 0.9602, test acc 0.9617\n",
      "fold 3, train loss 0.0007, valid loss 0.0008, train acc 0.9593, test acc 0.9585\n",
      "fold 4, train loss 0.0007, valid loss 0.0007, train acc 0.9596, test acc 0.9602\n",
      "fold 5, train loss 0.0007, valid loss 0.0008, train acc 0.9596, test acc 0.9596\n",
      "fold 6, train loss 0.0007, valid loss 0.0008, train acc 0.9594, test acc 0.9602\n",
      "fold 7, train loss 0.0007, valid loss 0.0008, train acc 0.9605, test acc 0.9613\n",
      "fold 8, train loss 0.0007, valid loss 0.0008, train acc 0.9613, test acc 0.9616\n",
      "fold 9, train loss 0.0007, valid loss 0.0008, train acc 0.9605, test acc 0.9609\n",
      "fold 10, train loss 0.0007, valid loss 0.0006, train acc 0.9592, test acc 0.9608\n"
     ]
    }
   ],
   "source": [
    "import torch.nn as nn\n",
    "from torch.nn import init\n",
    "import torch.utils.data\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "\n",
    "mnist_train = torchvision.datasets.MNIST(root='./Datasets/MNIST', train=True, download=True, transform=transforms.ToTensor())\n",
    "mnist_test = torchvision.datasets.MNIST(root='./Datasets/MNIST', train=False, download=True, transform=transforms.ToTensor())\n",
    "\n",
    "batch_size = 200\n",
    "train_iter = torch.utils.data.DataLoader(mnist_train, batch_size=batch_size, shuffle=True, num_workers=0)\n",
    "test_iter = torch.utils.data.DataLoader(mnist_test, batch_size=batch_size, shuffle=False,num_workers=0)\n",
    "\n",
    "a, b =[], []\n",
    "for x, y in mnist_train:\n",
    "    a.append(x)\n",
    "    b.append(y)\n",
    "\n",
    "x = torch.stack(a, dim=0)\n",
    "y = torch.tensor(b)\n",
    "\n",
    "num_inputs, num_outputs, num_hiddens = 784, 10, 256\n",
    "\n",
    "\n",
    "def evaluate_accuracy(test_iter, net):\n",
    "    acc_sum, n = 0, 0\n",
    "    for X, y in test_iter:\n",
    "        y_hat = net(X)\n",
    "        acc_sum += (y_hat.argmax(dim=1) == y).sum().item()\n",
    "        n += y.shape[0]\n",
    "    return acc_sum / n\n",
    "\n",
    "\n",
    "num_epochs = 12\n",
    "\n",
    "\n",
    "def train(x_train, y_train, x_valid, y_valid, num_epochs, batch_size):\n",
    "    train_ls, test_ls, x_epoch, list_train_acc, list_test_acc = [], [], [], [], []\n",
    "    train_dataset = torch.utils.data.TensorDataset(x_train,y_train)\n",
    "    train_dataiter = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True, num_workers=0)\n",
    "    valid_dataset = torch.utils.data.TensorDataset(x_valid,y_valid)\n",
    "    valid_dataiter = torch.utils.data.DataLoader(dataset=valid_dataset, batch_size=batch_size, shuffle=False, num_workers=0)\n",
    "\n",
    "    class FlattenLayer(nn.Module):\n",
    "        def __init__(self):\n",
    "            super(FlattenLayer, self).__init__()\n",
    "\n",
    "        def forward(self, x):\n",
    "            return x.view(x.shape[0], -1)\n",
    "\n",
    "    net = nn.Sequential(\n",
    "        FlattenLayer(),\n",
    "        nn.Linear(num_inputs, num_hiddens),\n",
    "        nn.ReLU(),  \n",
    "        nn.Linear(num_hiddens, num_outputs),\n",
    "    )\n",
    "\n",
    "    for params in net.parameters():\n",
    "        init.normal_(params, mean=0, std=0.1)\n",
    "\n",
    "    loss = torch.nn.CrossEntropyLoss()\n",
    "    optimizer = torch.optim.SGD(net.parameters(), lr=0.2)\n",
    "\n",
    "    for epoch in range(num_epochs):\n",
    "        train_1_sum, train_acc_sum, n = 0.0, 0.0, 0\n",
    "        train_1_test_sum, n_test = 0.0, 0\n",
    "        for X, y in train_dataiter:\n",
    "            y_hat = net(X)\n",
    "            l = loss(y_hat, y).sum()\n",
    "            optimizer.zero_grad()\n",
    "            l.backward()\n",
    "            optimizer.step()\n",
    "\n",
    "            train_1_sum += l.item()\n",
    "            train_acc_sum += (y_hat.argmax(dim=1) == y).sum().item()\n",
    "            n += y.shape[0]\n",
    "        train_ls.append(train_1_sum / n)\n",
    "        x_epoch.append(epoch + 1)\n",
    "        test_acc = evaluate_accuracy(test_iter, net)\n",
    "        list_train_acc.append(train_acc_sum / n)\n",
    "        list_test_acc.append(test_acc)\n",
    "        # print('epoch %d, loss %.4f, train acc %.3f, test acc %.3f' % (epoch + 1, train_1_sum / n, train_acc_sum / n, test_acc))\n",
    "\n",
    "        for X_test, y_test in valid_dataiter:\n",
    "            y_hat = net(X_test)\n",
    "            l = loss(y_hat, y_test).sum()\n",
    "            train_1_test_sum += l.item()\n",
    "            n_test += y_test.shape[0]\n",
    "        test_ls.append(train_1_test_sum / n_test)\n",
    "\n",
    "    return train_ls, test_ls, list_train_acc, list_test_acc\n",
    "\n",
    "\n",
    "def get_k_fold_data(k, i, X, y):\n",
    "    # 返回第i折交叉验证时所需要的训练和验证数据，分开放，X_train为训练数据，X_valid为验证数据\n",
    "    assert k > 1\n",
    "    fold_size = X.shape[0] // k  \n",
    "    X_train, y_train = None, None\n",
    "    for j in range(k):\n",
    "        idx = slice(j * fold_size, (j + 1) * fold_size)  #slice(start,end,step)切片函数\n",
    "        X_part, y_part = X[idx, :], y[idx]\n",
    "        if j == i:\n",
    "            X_valid, y_valid = X_part, y_part\n",
    "        elif X_train is None:\n",
    "            X_train, y_train = X_part, y_part\n",
    "        else:\n",
    "            X_train = torch.cat((X_train, X_part), dim=0)\n",
    "            y_train = torch.cat((y_train, y_part), dim=0)\n",
    "    return X_train, y_train, X_valid, y_valid\n",
    "\n",
    "\n",
    "def k_fold(k, X_train, y_train, num_epochs, batch_size):\n",
    "    for i in range(k):\n",
    "        data = get_k_fold_data(k, i, X_train, y_train) # 获取k折交叉验证的训练和验证数据\n",
    "        train_ls, valid_ls, train_acc, valid_acc = train(*data, num_epochs, batch_size)\n",
    "        train_l_sum, valid_l_sum, train_acc_sum, valid_acc_sum = 0.0, 0.0, 0.0, 0.0\n",
    "        for ii in train_ls:\n",
    "            train_l_sum += ii\n",
    "        for jj in valid_ls:\n",
    "            valid_l_sum += jj\n",
    "        for ii in train_acc:\n",
    "            train_acc_sum += ii\n",
    "        for jj in valid_acc:\n",
    "            valid_acc_sum += jj\n",
    "        print('fold %d, train loss %.4f, valid loss %.4f, train acc %.4f, test acc %.4f' % (i + 1, train_l_sum / num_epochs, valid_l_sum / num_epochs, train_acc_sum / num_epochs, valid_acc_sum / num_epochs))\n",
    "\n",
    "\n",
    "k_fold(10, x, y, num_epochs, batch_size)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}